#!/usr/bin/env ruby

# Copyright (c) 2010-2012 Microsoft Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#	http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

require 'rubygems'
require 'ruby-aws'
require 'optparse'
require 'digest/md5'
require 'rexml/document'

require './include/array'
require './include/assignment'
require './include/assignment_set'
require './include/config'
require './include/proxy'
require './include/requester'
require './include/templates'

@config = MOS::Config.create
@mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => @config[:Server]

@options = { }
optparse = OptionParser.new do |opts|
	opts.banner = "Usage: #{$0} [options]"
 
	@options[:matlab_output] = false
	opts.on('--matlab-output', 'Create MATLAB data files with raw scores' ) do
		@options[:matlab_output] = true
	end

	@options[:only_headphones] = false
	opts.on('--headphones', 'Only analyze requests submitted by users wearing headphones' ) do
		@options[:only_headphones] = true
	end

	@options[:no_headphones] = false
	opts.on('--no-headphones', 'Only analyze requests submitted by users NOT wearing headphones' ) do
		@options[:no_headphones] = true
	end

	@options[:commit] = false
	opts.on( '-c', '--commit', 'Approve or reject assignments, committing results to the Mechanical Turk server' ) do
		@options[:commit] = true
	end

	@options[:quiet] = false
	opts.on( '-q', '--quiet', 'Do not print any messages (implies -f)' ) do
		@options[:quiet] = true
	end

	@options[:force] = false
	opts.on( '-f', '--force', 'Do not ask for confirmation' ) do
		@options[:force] = true
	end
 
	opts.on( '-h', '--help', 'Display this screen' ) do
		puts opts
		exit
	end

	opts.on_tail("")
	opts.on_tail("This script automatically analyzes and accepts or rejects previously downloaded assignments.")
	opts.on_tail("Before running this script, use GetAssignments.rb to collect submitted assignments.")
end

begin
	optparse.parse!
rescue OptionParser::ParseError => e
	$stderr.puts("Error parsing command-line arguments: " + e.message)
	$stderr.puts("Try '#{$0} --help' for more information.")
	exit
end

if @options[:quiet]
	@options[:force] = true
	devnull = File.exist?('/dev/null') ? '/dev/null' : 'NUL'
	$stdout = File.new(devnull, 'w')
end

if @options[:commit] && !@options[:force]
	printf("Are you sure you want to approve/reject all collected assignments? (y/n) ")
	response = STDIN.gets.chomp.downcase
	if response != 'y'
		printf("Exiting...\n")
		exit
	end
end

#############
# Entry Point
#############

input = Amazon::Util::DataReader.load(@config[:HITInputFile], :Tabular)
input.each do |line|
	line[:sentence]   = line[:sentence].to_s
	line[:algorithm]  = line[:algorithm].to_s
	line[:identifier] = line[:sentence] + '_' + line[:algorithm]
end
input.sort!{ |a,b| a[:identifier] <=> b[:identifier] }
input.each_index { |i| input[i][:identifier] = 'ID' + i.to_s } if @config[:RewriteIdentifier]
input = MOS.indexByUniqueKey(input, :identifier)

if !File.exist?(@config[:HITResultsFile])
	$stderr.puts("No results file to analyze. Please run GetAssignments.rb first.")
	exit
end

results = Amazon::Util::DataReader.load(@config[:HITResultsFile], :Tabular)
assignment_set = results.clone
assignment_set.map! { |a| MOS::Assignment.new(a.clone, input) }
assignment_set = MOS::AssignmentSet.new(assignment_set)

fast_workers = assignment_set.fast_workers(@config[:SamplesPerHIT] * @config[:MinSampleWorkingTime])
bad_workers  = assignment_set.bad_workers(@config[:BonusMinAssignments])

printf("=======================\n")
printf("Fast Workers (rejected)\n")
printf("=======================\n\n")
assignment_set.print_worker_stats(fast_workers.keys)

printf("======================\n")
printf("Bad Workers (rejected)\n")
printf("======================\n\n")
assignment_set.print_worker_stats(bad_workers.keys - fast_workers.keys)

results.delete_if { |a| fast_workers[a[:WorkerId].to_sym] }
results.delete_if { |a| bad_workers [a[:WorkerId].to_sym] }
results.delete_if { |a| a[:headphones] != 'in-ear' && a[:headphones] != 'over-the-ear' } if @options[:only_headphones]
results.delete_if { |a| a[:headphones] == 'in-ear' || a[:headphones] == 'over-the-ear' } if @options[  :no_headphones]
assignment_set = results.clone
assignment_set.map! { |a| MOS::Assignment.new(a.clone, input) }
assignment_set = MOS::AssignmentSet.new(assignment_set)

outliers = assignment_set.outliers(@config[:BonusMinAssignments])

printf("=====================\n")
printf("Other Ignored Workers\n")
printf("=====================\n\n")
assignment_set.print_worker_stats(outliers.keys)

results.delete_if { |a| outliers[a[:WorkerId].to_sym] }
assignment_set = results.clone
assignment_set.map! { |a| MOS::Assignment.new(a.clone, input) }
assignment_set = MOS::AssignmentSet.new(assignment_set)

printf("===============================================\n")
printf("Final Results (excluding all previous outliers)\n")
printf("===============================================\n\n")
assignment_set.print_algorithm_stats
assignment_set.print_sentence_stats
assignment_set.print_worker_stats

if assignment_set.mean_working_time != nil
	printf("Mean time spent per HIT: %d seconds\n", assignment_set.mean_working_time)
end

############################
# Approve/Reject Assignments
############################

results = Amazon::Util::DataReader.load(@config[:HITResultsFile], :Tabular)
results.each do |a|
	if a[:Status] != 'Approved' && a[:Status] != 'Rejected'
		approved = true
		reason   = 'thank you for participating in our study'

		if a[:WorkingTime] < @config[:SamplesPerHIT] * @config[:MinSampleWorkingTime]
			approved = false
			reason   = 'assignment was submitted too quickly to be accurate'
		end

		if bad_workers[a[:WorkerId].to_sym]
			approved = false
			reason   = 'assignment is significantly inaccurate'
		end

		if approved
			begin
				if @options[:commit]
					@mturk.approve_assignment(a[:AssignmentId],
							'Assignment approved: ' + reason)
					a[:Status] = 'Approved'
				end
				printf("Assignment %s approved: %s.\n", a[:AssignmentId], reason)
			rescue => e
				$stderr.puts("Error: " + e.message)
			end
		else
			begin
				if @options[:commit]
					@mturk.reject_assignment(a[:AssignmentId],
							'Assignment rejected: ' + reason)
					a[:Status] = 'Rejected'
				end
				printf("Assignment %s rejected: %s.\n", a[:AssignmentId], reason)
			rescue => e
				$stderr.puts("Error: " + e.message)
			end
		end
	end
end

if @options[:commit]
	if File.exist?(@config[:HITResultsFile])
		FileUtils.move(@config[:HITResultsFile], @config[:HITResultsFile] + '.backup')
	end
	Amazon::Util::DataReader.save(@config[:HITResultsFile], results, :Tabular)
end

######################
# Write MATLAB outputs
######################
assignment_set.print_raw_scores() if @options[:matlab_output]

